{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 130,
   "source": [
    "# timing\r\n",
    "from time import perf_counter\r\n",
    "# used for saving datas\r\n",
    "import time\r\n",
    "import os"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "source": [
    "# numpy provides arrays and useful functions for working with them\r\n",
    "import numpy\r\n",
    "\r\n",
    "import cupy\r\n",
    "import cupyx.scipy.ndimage"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "source": [
    "class neuralNetwork:\r\n",
    "    \"\"\"neural network class definition\"\"\"\r\n",
    "    \r\n",
    "    \r\n",
    "    def __init__(self, inputnodes, hiddennodes, outputnodes, learningrate):\r\n",
    "        \"\"\"initialise the neural network\"\"\"\r\n",
    "        # set number of nodes in each input, hidden, output layer\r\n",
    "        self.inodes = inputnodes\r\n",
    "        self.hnodes = hiddennodes\r\n",
    "        self.onodes = outputnodes\r\n",
    "        \r\n",
    "        # link weight matrices, wih and who\r\n",
    "        # weights inside the arrays are w_i_j, where link is from node i to node i to j in the next layer\r\n",
    "        # w11 w21\r\n",
    "        # w12 w22 etc\r\n",
    "        self.wih = cupy.random.normal(0.0, pow(self.inodes, -0.5), (self.hnodes, self.inodes))\r\n",
    "        self.who = cupy.random.normal(0.0, pow(self.hnodes, -0.5), (self.onodes, self.hnodes))\r\n",
    "        \r\n",
    "        #learning rate\r\n",
    "        self.lr = learningrate\r\n",
    "        \r\n",
    "        #activation function is the sigmoid function\r\n",
    "        self.activation_function = lambda x: 1 / (1 + cupy.exp(x) ** (-1))\r\n",
    "        \r\n",
    "        pass\r\n",
    "    \r\n",
    "    \r\n",
    "    def train(self, inputs_list, targets_list):\r\n",
    "        \"\"\"train the neural network\"\"\"\r\n",
    "        # convert inputs list to 2d array\r\n",
    "        inputs = cupy.array(inputs_list, ndmin=2).T\r\n",
    "        targets = cupy.array(targets_list, ndmin=2).T\r\n",
    "\r\n",
    "        # calculate signals into hidden layer\r\n",
    "        hidden_inputs = cupy.dot(self.wih, inputs)\r\n",
    "        # calculate the signals emerging from hidden layer\r\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\r\n",
    "\r\n",
    "        # calcute signals into final output layer\r\n",
    "        final_inputs = cupy.dot(self.who, hidden_outputs)\r\n",
    "        # calculate signals into final output layer\r\n",
    "        final_outputs = self.activation_function(final_inputs)\r\n",
    "\r\n",
    "        # output layer error is the (target - actual)\r\n",
    "        output_errors = targets - final_outputs\r\n",
    "        # hidden layer error is the output_errors, split by weights, recombined at hidden nodes\r\n",
    "        hidden_errors = cupy.dot(self.who.T, output_errors)\r\n",
    "\r\n",
    "        # update the weights for the links between the hidden and output layers\r\n",
    "        # S' = S(1 - S)\r\n",
    "        # dE / Dw = K * e * o * (1 - o)\r\n",
    "        self.who += self.lr * cupy.dot((output_errors * final_outputs * (1.0 - final_outputs)), cupy.transpose(hidden_outputs))\r\n",
    "        self.wih += self.lr * cupy.dot((hidden_errors * hidden_outputs * (1.0 - hidden_outputs)), cupy.transpose(inputs))\r\n",
    "\r\n",
    "        pass\r\n",
    "\r\n",
    "\r\n",
    "    def query(self, inputs_list):\r\n",
    "        \"\"\"query the neural network\"\"\"\r\n",
    "        # convert inputs list to 2d array\r\n",
    "        inputs = cupy.array(inputs_list, ndmin=2).T\r\n",
    "\r\n",
    "        # calculate signals into hidden layer\r\n",
    "        hidden_inputs = cupy.dot(self.wih, inputs)\r\n",
    "        # calculate the signals emerging from hidden layer\r\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\r\n",
    "\r\n",
    "        # calcute signals into final output layer\r\n",
    "        final_inputs = cupy.dot(self.who, hidden_outputs)\r\n",
    "        # calculate the signals emerging from final output layer\r\n",
    "        final_outputs = self.activation_function(final_inputs)\r\n",
    "\r\n",
    "        return final_outputs"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "source": [
    "def save_w(wih, who, hnodes=400):\r\n",
    "    print(\"Saving Data...\")\r\n",
    "    \r\n",
    "    t = time.asctime(time.localtime(time.time()))\r\n",
    "    t = t.replace(':', '_')\r\n",
    "\r\n",
    "    filepath = \"saved_data/\" + t + \"/\"\r\n",
    "    os.mkdir(filepath)\r\n",
    "    filename_wih = f\"hn{hnodes}_wih_epoch_{e + 1}.csv\"\r\n",
    "    filename_who = f\"hn{hnodes}_who_epoch_{e + 1}.csv\"\r\n",
    "    numpy.savetxt(filepath + filename_wih, wih)\r\n",
    "    numpy.savetxt(filepath + filename_who, who)\r\n",
    "    print(\"Saving Succeeded!\")\r\n",
    "    pass\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "source": [
    "# start counting\r\n",
    "start = perf_counter()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "source": [
    "# number of input, hidden and output nodes\r\n",
    "input_nodes = 784\r\n",
    "hidden_nodes = 400\r\n",
    "output_nodes = 10\r\n",
    "\r\n",
    "# learning rate\r\n",
    "learning_rate = 0.01\r\n",
    "\r\n",
    "# create instance of neural network\r\n",
    "n = neuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "source": [
    "# load the mnist training data CSV file into a list\r\n",
    "train_path = \"F:/Documents/mnist_datasets/mnist_train.csv\"\r\n",
    "training_data_file = open(train_path, 'r')\r\n",
    "training_data_list = training_data_file.readlines()\r\n",
    "training_data_file.close()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "source": [
    "# train the neural network\r\n",
    "\r\n",
    "print(\"Training...\")\r\n",
    "\r\n",
    "# epochs is the number of times the training data set is used for training\r\n",
    "epochs = 5\r\n",
    "\r\n",
    "for e in range(epochs):\r\n",
    "    print(\"Epoch:\", e)\r\n",
    "    # go through all records in the training data set\r\n",
    "    # count = 1\r\n",
    "    for record in training_data_list:\r\n",
    "        # print(f\"\\rRound: {count}\", end=\"\")\r\n",
    "        # count += 1\r\n",
    "        # split the tecord by the ',' commas\r\n",
    "        all_values = record.split(',')\r\n",
    "        temp = list(map(eval, all_values[1:]))\r\n",
    "        # scale and shift the inputs\r\n",
    "        inputs = (cupy.asarray(temp) / 255 * 0.99) + 0.01\r\n",
    "        # create the target output values (all 0.01, except the desired lavel which is 0.99)\r\n",
    "        targets = cupy.zeros(output_nodes) + 0.01\r\n",
    "        # all_values[0] is the target lavel for this record\r\n",
    "        targets[int(all_values[0])] = 0.99\r\n",
    "        # print(type(inputs))\r\n",
    "        # print(type(targets))\r\n",
    "        # input(\"\")\r\n",
    "        n.train(inputs, targets)\r\n",
    "        \r\n",
    "        # create rotated variations\r\n",
    "        # rotated anticlockwise by x degrees\r\n",
    "        \r\n",
    "        inputs_plusx_img = cupyx.scipy.ndimage.interpolation.rotate(inputs.reshape(28, 28), 10, cval=0.01, order=1, reshape=False)\r\n",
    "        n.train(inputs_plusx_img.reshape(784), targets)\r\n",
    "        # rotated clockwise by x degrees\r\n",
    "        inputs_minusx_img = cupyx.scipy.ndimage.interpolation.rotate(inputs.reshape(28, 28), -10, cval=0.01, order=1, reshape=False)\r\n",
    "        n.train(inputs_minusx_img.reshape(784), targets)\r\n",
    "        \r\n",
    "    # print(\"\")\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Training...\n",
      "Epoch: 0\n",
      "Epoch: 1\n",
      "Epoch: 2\n",
      "Epoch: 3\n",
      "Epoch: 4\n"
     ]
    }
   ],
   "metadata": {
    "scrolled": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "source": [
    "# load the mnist test data CSV file into a list\r\n",
    "test_path = \"F:/Documents/mnist_datasets/mnist_test.csv\"\r\n",
    "test_data_file = open(test_path, 'r')\r\n",
    "test_data_list = test_data_file.readlines()\r\n",
    "test_data_file.close()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "source": [
    "# test the neural network\r\n",
    "\r\n",
    "print(\"Testing...\")\r\n",
    "\r\n",
    "# scorecard for how well the network performs, initially empty\r\n",
    "scorecard = []\r\n",
    "\r\n",
    "# go through all the records in the test data set\r\n",
    "for record in test_data_list:\r\n",
    "    # split the record by the ',' commas\r\n",
    "    all_values = record.split(',')\r\n",
    "    # correct answer is first value\r\n",
    "    correct_label = int(all_values[0])\r\n",
    "    # scale and shift the inputs\r\n",
    "    temp = list(map(eval, all_values[1:]))\r\n",
    "    inputs = (cupy.asarray(temp) / 255.0 * 0.99) + 0.01\r\n",
    "    # query the network\r\n",
    "    outputs = n.query(inputs)\r\n",
    "    # the index of the highest value corresponds to the label\r\n",
    "    label = cupy.argmax(outputs)\r\n",
    "    # append correct or incorrect to list\r\n",
    "    if (label == correct_label):\r\n",
    "        # network's answer matches correct answer, add 1 to scorecard\r\n",
    "        scorecard.append(1)\r\n",
    "    else:\r\n",
    "        # network's answer doesn't match correct answer, add 0 to scorecard\r\n",
    "        scorecard.append(0)\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Testing...\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "source": [
    "save_w(cupy.asnumpy(n.wih), cupy.asnumpy(n.who), hidden_nodes)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving Data...\n",
      "Saving Succeeded!\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "source": [
    "# calculate the performance score, the fraction of correct answers\r\n",
    "scorecard_array = cupy.asarray(scorecard)\r\n",
    "print(\"performance =\", scorecard_array.sum() / scorecard_array.size)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "performance = 0.9717\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "source": [
    "# stop counting\r\n",
    "end = perf_counter()\r\n",
    "dur = end - start\r\n",
    "mins = (end - start) // 60\r\n",
    "secs = dur % 60\r\n",
    "print(\"I'v spent {:.00f} mins {:.00f} secs on this.\".format(mins, secs))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "I'v spent 42 mins 21 secs on this.\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.4 64-bit ('base': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "interpreter": {
   "hash": "a8f61be024eba58adef938c9aa1e29e02cb3dece83a5348b1a2dafd16a070453"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}